{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "import os,sys\n",
    "sys.path.insert(0,\"..\")\n",
    "from glob import glob\n",
    "from os.path import exists, join\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import argparse\n",
    "\n",
    "import torch\n",
    "import torchvision, torchvision.transforms\n",
    "import skimage.transform\n",
    "import sklearn\n",
    "\n",
    "import random\n",
    "import train_utils\n",
    "import pickle\n",
    "import torchxrayvision as xrv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "results_chex-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_chex.pkl\r\n",
      "results_kaggle-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_kaggle.pkl\r\n",
      "results_mimic_ch-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_mimic_ch.pkl\r\n",
      "results_mimic_nb-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_mimic_nb.pkl\r\n",
      "results_nih-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_nih.pkl\r\n",
      "results_nih-pc-chex-mimic_ch-google-openi-kaggle-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_nih-pc-chex-mimic_ch-google-openi-kaggle.pkl\r\n",
      "results_pc-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_pc.pkl\r\n"
     ]
    }
   ],
   "source": [
    "!ls res*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"nih-pc-chex-mimic_ch-google-openi-kaggle\"\n",
    "results = pickle.load(open(\"results_{}-densenet121-d121-tw-lr001-rot45-tr15-sc15-seed0-best_{}.pkl\".format(dataset_name, dataset_name), \"br\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_threshs = []\n",
    "all_min = []\n",
    "all_max = []\n",
    "all_ppv80 = []\n",
    "for i, patho in enumerate(xrv.datasets.default_pathologies):\n",
    "    opt_thres = np.nan\n",
    "    opt_max = np.nan\n",
    "    if len(results[3][i]) > 0:\n",
    "        \n",
    "        #sigmoid\n",
    "        all_outputs = 1.0/(1.0 + np.exp(-results[2][i]))\n",
    "        \n",
    "        fpr, tpr, thres = sklearn.metrics.roc_curve(results[3][i], all_outputs)\n",
    "        pente = tpr - fpr\n",
    "        opt_thres = thres[np.argmax(pente)]\n",
    "        opt_min = all_outputs.min()\n",
    "        opt_max = all_outputs.max()\n",
    "        \n",
    "        ppv, recall, thres = sklearn.metrics.precision_recall_curve(results[3][i], all_outputs)\n",
    "        ppv80_thres_idx = np.where(ppv > 0.8)[0][0]\n",
    "        ppv80_thres = thres[ppv80_thres_idx-1]\n",
    "        \n",
    "        \n",
    "    all_threshs.append(opt_thres)\n",
    "    all_min.append(opt_min)\n",
    "    all_max.append(opt_max)\n",
    "    all_ppv80.append(ppv80_thres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.07422872, 0.038290843, 0.09814756, 0.0098118475, 0.023601074, 0.0022490358, 0.010060724, 0.103246614, 0.056810737, 0.026791653, 0.050318155, 0.023985857, 0.01939503, 0.042889766, 0.053369623, 0.035975814, 0.20204692, 0.05015312]\n"
     ]
    }
   ],
   "source": [
    "print(str(all_threshs).replace(\"nan\",\"np.nan\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.9620523e-05, 1.8223789e-05, 0.00051482924, 1.8099824e-07, 5.410019e-09, 8.306624e-09, 1.9041477e-09, 1.1754888e-05, 9.194446e-07, 3.6992893e-07, 5.6234506e-07, 5.929607e-05, 1.1718594e-06, 1.2792777e-08, 5.942449e-07, 1.1029978e-05, 0.0006902651, 9.664481e-05]\n"
     ]
    }
   ],
   "source": [
    "print(str(all_min).replace(\"nan\",\"np.nan\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.9676907, 0.9329898, 0.9454921, 0.691005, 0.95700103, 0.46127197, 0.8549093, 0.99547607, 0.9962031, 0.6920735, 0.9889571, 0.8832056, 0.9939653, 0.9999714, 0.9900496, 0.90185666, 0.9943041, 0.9523041]\n"
     ]
    }
   ],
   "source": [
    "print(str(all_max).replace(\"nan\",\"np.nan\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.72715247, 0.8885005, 0.92493945, 0.6527224, 0.68707734, 0.46127197, 0.7272054, 0.6127343, 0.9878492, 0.61979693, 0.66309816, 0.7853459, 0.930661, 0.93645346, 0.6788558, 0.6547198, 0.61614525, 0.8489876]\n"
     ]
    }
   ],
   "source": [
    "print(str(all_ppv80).replace(\"nan\",\"np.nan\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "ppv, recall, thres = sklearn.metrics.precision_recall_curve(results[3][0], results[2][0], pos_label=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "ppv80_thres = np.where(ppv > 0.8)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "npv80_thres = np.where(npv > 0.8)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98024523"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thres[ppv80_thres]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7",
   "language": "python",
   "name": "python3.7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
